#!/bin/sh
# Bind Chelsio interrupt threads to different cores (melifaro@FreeBSD.org)
# If numa-domain is present, bind in the same numa-domain

# PROVIDE:      chelsio_affinity
# REQUIRE:      FILESYSTEMS netif
# KEYWORD:      nojail

#
# Add the following lines to /etc/rc.conf.local or /etc/rc.conf
# to enable this service:
#
# chelsio_affinity_enable (bool):   Set to YES by default.
#               Set it to YES to bind interrupt threads to different cores
# chelsio_affinity_avoidncpu (int):  0 (none) by default.
#	            Set to number of CPU to exclude from binding
#				This to avoid overloading all CPUs in case of huge
#				traffic received

. /etc/rc.subr

name="chelsio_affinity"
rcvar=${name}_enable

start_cmd="chelsio_affinity_start"

bind_ports() {
	# card_type example: t5nex|t6nex
	# card_index example: 0|1
	card_type=$1
	card_index=$2
	card=${card_type}${card_index}
	dev=dev.${card_type}.${card_index}
	card_domain=$(/sbin/sysctl -n ${dev}.%domain 2> /dev/null)
	if [ -z $card_domain ]; then
		echo "No %domain found for ${dev}.%domain, setting to 0"
		card_domain=0
	fi

	echo "$card on numa-domain ${card_domain}"
	# find ports
	ports=$(/sbin/sysctl -n ${dev}.nports 2> /dev/null)
	if [ -n $ports ]; then
		echo " $card has $ports ports"
		# Iterate through ports on this card
		for port in $(seq 0 $((ports - 1))); do
			echo "  $card port $port"
			# List all CPUs from this domain
			# cpuset output example: 0 1 2 3 4 5 6 7 8 9 10 11
			CPUS=$(/usr/bin/cpuset -g -d $card_domain | sed 's/domain.*mask: //g;s/\,//g')
			set $CPUS
			if [ ${chelsio_affinity_avoidncpu} -ne 0 ]; then
				for avoidcpu in $(seq 0 $((chelsio_affinity_avoidncpu - 1))); do
					shift
				done
			fi
			for irq in $(/usr/bin/vmstat -ai |\
				/usr/bin/sed -nE "/${card}:${port}a/s/irq([[:digit:]]+):.*/\1/p"); do
				if [ -z "$1" ]; then
					# case when there are less CPU than IRQ/Queue (can this be possible?)
					echo "    Warning: Less available CPU than queues to bind"
					set $CPUS
				fi
				echo "   Bind ${card}:${port}a IRQ ${irq} to CPU $1"
				/usr/bin/cpuset -l $1 -x ${irq}
				shift
			done
		done
	else
		echo "$card has no ports!"
		exit 0
	fi
}

chelsio_affinity_start() {
    # Need a Chelsio Card, search for T5 and T6 cards
    Nt5=$(/sbin/sysctl -i dev.t5nex | grep -c nports)
    Nt6=$(/sbin/sysctl -i dev.t6nex | grep -c nports)

    if [ $Nt5 -eq 0 ] && [ $Nt6 -eq 0 ]; then
        echo "No Chelsio card detected"
        exit 0
    fi
    echo "# of T5 cards: $Nt5, # of T6 cards: $Nt6"

    # Minimum of 2 CPU mandatory
    NCPU=$(/sbin/sysctl -n hw.ncpu)
    if [ ${NCPU} -le 2 ]; then
        echo "Need 2 CPU minimum"
        exit 0
    fi

    # iterate through cards and bind queues on every card/port/queue combo to cpus
    if [ $Nt5 -gt 0 ]; then
        for j in $(seq 0 $((Nt5 - 1))); do
            bind_ports t5nex $j
        done
    fi
    if [ $Nt6 -gt 0 ]; then
        for j in $(seq 0 $((Nt6 - 1))); do
            bind_ports t6nex $j
        done
    fi
}

load_rc_config $name

: ${chelsio_affinity_enable="NO"}
: ${chelsio_affinity_avoidncpu=0}

run_rc_command "$1"
